{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# import relevant packages\n",
    "from keras.datasets import mnist\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.utils import np_utils\n",
    "from keras.layers import Flatten\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.convolutional import MaxPooling2D\n",
    "from keras.utils import np_utils\n",
    "from keras import backend as K\n",
    "from keras import regularizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create a simple dataset\n",
    "X_train=np.array([[[1,2,3,4],[2,3,4,5],[5,6,7,8],[1,3,4,5]],[[-1,2,3,-4],[2,-3,4,5],[-5,6,-7,8],[-1,-3,-4,-5]]])\n",
    "y_train=np.array([0,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Normalize the inputs by dividing each value with the maximum value in dataset\n",
    "X_train = X_train / 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.125, 0.25 , 0.375, 0.5  ],\n",
       "       [0.25 , 0.375, 0.5  , 0.625],\n",
       "       [0.625, 0.75 , 0.875, 1.   ],\n",
       "       [0.125, 0.375, 0.5  , 0.625]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# One hot encode the outputs\n",
    "y_train = np_utils.to_categorical(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 0.])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = X_train.reshape(X_train.shape[0],X_train.shape[1],X_train.shape[1],1 ).astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 4, 4, 1)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_6 (Conv2D)            (None, 2, 2, 1)           10        \n",
      "_________________________________________________________________\n",
      "max_pooling2d_6 (MaxPooling2 (None, 1, 1, 1)           0         \n",
      "_________________________________________________________________\n",
      "flatten_6 (Flatten)          (None, 1)                 0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 10)                20        \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 2)                 22        \n",
      "=================================================================\n",
      "Total params: 52\n",
      "Trainable params: 52\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(1, (3,3), input_shape=(4,4,1), activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(10, activation='relu'))\n",
    "model.add(Dense(2, activation='softmax'))\n",
    "# Compile model\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.4062 - acc: 1.0000\n",
      "Epoch 2/100\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.4037 - acc: 1.0000\n",
      "Epoch 3/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.4012 - acc: 1.0000\n",
      "Epoch 4/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3986 - acc: 1.0000\n",
      "Epoch 5/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3961 - acc: 1.0000\n",
      "Epoch 6/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3935 - acc: 1.0000\n",
      "Epoch 7/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3911 - acc: 1.0000\n",
      "Epoch 8/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3886 - acc: 1.0000\n",
      "Epoch 9/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3861 - acc: 1.0000\n",
      "Epoch 10/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3835 - acc: 1.0000\n",
      "Epoch 11/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3811 - acc: 1.0000\n",
      "Epoch 12/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3785 - acc: 1.0000\n",
      "Epoch 13/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3760 - acc: 1.0000\n",
      "Epoch 14/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3736 - acc: 1.0000\n",
      "Epoch 15/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3711 - acc: 1.0000\n",
      "Epoch 16/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3687 - acc: 1.0000\n",
      "Epoch 17/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3661 - acc: 1.0000\n",
      "Epoch 18/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3639 - acc: 1.0000\n",
      "Epoch 19/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3614 - acc: 1.0000\n",
      "Epoch 20/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3590 - acc: 1.0000\n",
      "Epoch 21/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3566 - acc: 1.0000\n",
      "Epoch 22/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3543 - acc: 1.0000\n",
      "Epoch 23/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3519 - acc: 1.0000\n",
      "Epoch 24/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3495 - acc: 1.0000\n",
      "Epoch 25/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3471 - acc: 1.0000\n",
      "Epoch 26/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3449 - acc: 1.0000\n",
      "Epoch 27/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3426 - acc: 1.0000\n",
      "Epoch 28/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3402 - acc: 1.0000\n",
      "Epoch 29/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3378 - acc: 1.0000\n",
      "Epoch 30/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3358 - acc: 1.0000\n",
      "Epoch 31/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3336 - acc: 1.0000\n",
      "Epoch 32/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3313 - acc: 1.0000\n",
      "Epoch 33/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3289 - acc: 1.0000\n",
      "Epoch 34/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3267 - acc: 1.0000\n",
      "Epoch 35/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3246 - acc: 1.0000\n",
      "Epoch 36/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3224 - acc: 1.0000\n",
      "Epoch 37/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3202 - acc: 1.0000\n",
      "Epoch 38/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3179 - acc: 1.0000\n",
      "Epoch 39/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3157 - acc: 1.0000\n",
      "Epoch 40/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3137 - acc: 1.0000\n",
      "Epoch 41/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3115 - acc: 1.0000\n",
      "Epoch 42/100\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.3095 - acc: 1.0000\n",
      "Epoch 43/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3074 - acc: 1.0000\n",
      "Epoch 44/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3052 - acc: 1.0000\n",
      "Epoch 45/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3032 - acc: 1.0000\n",
      "Epoch 46/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.3012 - acc: 1.0000\n",
      "Epoch 47/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2992 - acc: 1.0000\n",
      "Epoch 48/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2973 - acc: 1.0000\n",
      "Epoch 49/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2952 - acc: 1.0000\n",
      "Epoch 50/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2931 - acc: 1.0000\n",
      "Epoch 51/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2913 - acc: 1.0000\n",
      "Epoch 52/100\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.2894 - acc: 1.0000\n",
      "Epoch 53/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2874 - acc: 1.0000\n",
      "Epoch 54/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2855 - acc: 1.0000\n",
      "Epoch 55/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2835 - acc: 1.0000\n",
      "Epoch 56/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2817 - acc: 1.0000\n",
      "Epoch 57/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2799 - acc: 1.0000\n",
      "Epoch 58/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2780 - acc: 1.0000\n",
      "Epoch 59/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2762 - acc: 1.0000\n",
      "Epoch 60/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2744 - acc: 1.0000\n",
      "Epoch 61/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2728 - acc: 1.0000\n",
      "Epoch 62/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2710 - acc: 1.0000\n",
      "Epoch 63/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2690 - acc: 1.0000\n",
      "Epoch 64/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2672 - acc: 1.0000\n",
      "Epoch 65/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2656 - acc: 1.0000\n",
      "Epoch 66/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2638 - acc: 1.0000\n",
      "Epoch 67/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2622 - acc: 1.0000\n",
      "Epoch 68/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2605 - acc: 1.0000\n",
      "Epoch 69/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2588 - acc: 1.0000\n",
      "Epoch 70/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2573 - acc: 1.0000\n",
      "Epoch 71/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2556 - acc: 1.0000\n",
      "Epoch 72/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2541 - acc: 1.0000\n",
      "Epoch 73/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2525 - acc: 1.0000\n",
      "Epoch 74/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2509 - acc: 1.0000\n",
      "Epoch 75/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2493 - acc: 1.0000\n",
      "Epoch 76/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2477 - acc: 1.0000\n",
      "Epoch 77/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2464 - acc: 1.0000\n",
      "Epoch 78/100\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.2449 - acc: 1.0000\n",
      "Epoch 79/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2433 - acc: 1.0000\n",
      "Epoch 80/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2418 - acc: 1.0000\n",
      "Epoch 81/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2403 - acc: 1.0000\n",
      "Epoch 82/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2388 - acc: 1.0000\n",
      "Epoch 83/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2373 - acc: 1.0000\n",
      "Epoch 84/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2360 - acc: 1.0000\n",
      "Epoch 85/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2345 - acc: 1.0000\n",
      "Epoch 86/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2331 - acc: 1.0000\n",
      "Epoch 87/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2317 - acc: 1.0000\n",
      "Epoch 88/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2304 - acc: 1.0000\n",
      "Epoch 89/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2289 - acc: 1.0000\n",
      "Epoch 90/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2278 - acc: 1.0000\n",
      "Epoch 91/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2265 - acc: 1.0000\n",
      "Epoch 92/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2252 - acc: 1.0000\n",
      "Epoch 93/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2239 - acc: 1.0000\n",
      "Epoch 94/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2225 - acc: 1.0000\n",
      "Epoch 95/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2213 - acc: 1.0000\n",
      "Epoch 96/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2200 - acc: 1.0000\n",
      "Epoch 97/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2187 - acc: 1.0000\n",
      "Epoch 98/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2174 - acc: 1.0000\n",
      "Epoch 99/100\n",
      "2/2 [==============================] - 0s 2ms/step - loss: 0.2163 - acc: 1.0000\n",
      "Epoch 100/100\n",
      "2/2 [==============================] - 0s 3ms/step - loss: 0.2150 - acc: 1.0000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f97fdbd6d68>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, epochs=100, batch_size=2, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<keras.layers.convolutional.Conv2D at 0x7f97fe6e16d8>,\n",
       " <keras.layers.pooling.MaxPooling2D at 0x7f97fe6e1748>,\n",
       " <keras.layers.core.Flatten at 0x7f97fe748780>,\n",
       " <keras.layers.core.Dense at 0x7f97fe734f98>,\n",
       " <keras.layers.core.Dense at 0x7f97fe7340f0>]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[[[ 0.6502779 ]],\n",
       " \n",
       "         [[ 0.3674555 ]],\n",
       " \n",
       "         [[-0.04364061]]],\n",
       " \n",
       " \n",
       "        [[[ 0.8205539 ]],\n",
       " \n",
       "         [[ 0.5735873 ]],\n",
       " \n",
       "         [[ 0.13939373]]],\n",
       " \n",
       " \n",
       "        [[[-0.1292512 ]],\n",
       " \n",
       "         [[ 0.05793982]],\n",
       " \n",
       "         [[-0.03353929]]]], dtype=float32),\n",
       " array([-0.11086546], dtype=float32)]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[0].get_weights()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "conv2d_6/kernel:0 (3, 3, 1, 1)\n",
      "conv2d_6/bias:0 (1,)\n",
      "dense_11/kernel:0 (1, 10)\n",
      "dense_11/bias:0 (10,)\n",
      "dense_12/kernel:0 (10, 2)\n",
      "dense_12/bias:0 (2,)\n"
     ]
    }
   ],
   "source": [
    "names = [weight.name for layer in model.layers for weight in layer.weights]\n",
    "weights = model.get_weights()\n",
    "\n",
    "for name, weight in zip(names, weights):\n",
    "    print(name, weight.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.8906642 , 0.10933578]], dtype=float32)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(X_train[0].reshape(1,4,4,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
